<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">

<html lang="en">

<head>
  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  <title>LCOV - code analysis - /disk1/DNNAcc/gemmlowp/fixedpoint/fixedpoint.h</title>
  <link rel="stylesheet" type="text/css" href="../../../../gcov.css">
</head>

<body>

  <table width="100%" border=0 cellspacing=0 cellpadding=0>
    <tr><td class="title">LCOV - code coverage report</td></tr>
    <tr><td class="ruler"><img src="../../../../glass.png" width=3 height=3 alt=""></td></tr>

    <tr>
      <td width="100%">
        <table cellpadding=1 border=0 width="100%">
          <tr>
            <td width="10%" class="headerItem">Current view:</td>
            <td width="35%" class="headerValue"><a href="../../../../index.html">top level</a> - <a href="index.html">disk1/DNNAcc/gemmlowp/fixedpoint</a> - fixedpoint.h<span style="font-size: 80%;"> (source / <a href="fixedpoint.h.func-sort-c.html">functions</a>)</span></td>
            <td width="5%"></td>
            <td width="15%"></td>
            <td width="10%" class="headerCovTableHead">Hit</td>
            <td width="10%" class="headerCovTableHead">Total</td>
            <td width="15%" class="headerCovTableHead">Coverage</td>
          </tr>
          <tr>
            <td class="headerItem">Test:</td>
            <td class="headerValue">code analysis</td>
            <td></td>
            <td class="headerItem">Lines:</td>
            <td class="headerCovTableEntry">235</td>
            <td class="headerCovTableEntry">235</td>
            <td class="headerCovTableEntryHi">100.0 %</td>
          </tr>
          <tr>
            <td class="headerItem">Date:</td>
            <td class="headerValue">2020-09-17 13:31:02</td>
            <td></td>
            <td class="headerItem">Functions:</td>
            <td class="headerCovTableEntry">618</td>
            <td class="headerCovTableEntry">618</td>
            <td class="headerCovTableEntryHi">100.0 %</td>
          </tr>
          <tr>
            <td class="headerItem">Legend:</td>
            <td class="headerValueLeg">            Lines:
            <span class="coverLegendCov">hit</span>
            <span class="coverLegendNoCov">not hit</span>
</td>
            <td></td>
          </tr>
          <tr><td><img src="../../../../glass.png" width=3 height=3 alt=""></td></tr>
        </table>
      </td>
    </tr>

    <tr><td class="ruler"><img src="../../../../glass.png" width=3 height=3 alt=""></td></tr>
  </table>

  <table cellpadding=0 cellspacing=0 border=0>
    <tr>
      <td><br></td>
    </tr>
    <tr>
      <td>
<pre class="sourceHeading">          Line data    Source code</pre>
<pre class="source">
<a name="1"><span class="lineNum">       1 </span>            : // Copyright 2015 The Gemmlowp Authors. All Rights Reserved.</a>
<span class="lineNum">       2 </span>            : //
<span class="lineNum">       3 </span>            : // Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);
<span class="lineNum">       4 </span>            : // you may not use this file except in compliance with the License.
<span class="lineNum">       5 </span>            : // You may obtain a copy of the License at
<span class="lineNum">       6 </span>            : //
<span class="lineNum">       7 </span>            : //     http://www.apache.org/licenses/LICENSE-2.0
<span class="lineNum">       8 </span>            : //
<span class="lineNum">       9 </span>            : // Unless required by applicable law or agreed to in writing, software
<span class="lineNum">      10 </span>            : // distributed under the License is distributed on an &quot;AS IS&quot; BASIS,
<span class="lineNum">      11 </span>            : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<span class="lineNum">      12 </span>            : // See the License for the specific language governing permissions and
<span class="lineNum">      13 </span>            : // limitations under the License.
<span class="lineNum">      14 </span>            : 
<span class="lineNum">      15 </span>            : // fixedpoint.h: fixed-point arithmetic, with basic operations and
<span class="lineNum">      16 </span>            : // a few math functions such as tanh.
<span class="lineNum">      17 </span>            : 
<span class="lineNum">      18 </span>            : #ifndef GEMMLOWP_INTERNAL_FIXEDPOINT_H_
<span class="lineNum">      19 </span>            : #define GEMMLOWP_INTERNAL_FIXEDPOINT_H_
<span class="lineNum">      20 </span>            : 
<span class="lineNum">      21 </span>            : #include &lt;algorithm&gt;
<span class="lineNum">      22 </span>            : #include &lt;cassert&gt;
<span class="lineNum">      23 </span>            : #include &lt;cmath&gt;
<span class="lineNum">      24 </span>            : #include &lt;cstdint&gt;
<span class="lineNum">      25 </span>            : #include &lt;limits&gt;
<span class="lineNum">      26 </span>            : 
<span class="lineNum">      27 </span>            : #include &quot;../internal/detect_platform.h&quot;
<span class="lineNum">      28 </span>            : 
<span class="lineNum">      29 </span>            : namespace gemmlowp {
<span class="lineNum">      30 </span>            : 
<span class="lineNum">      31 </span>            : // Part 1: Low-level integer-arithmetic primitives.
<span class="lineNum">      32 </span>            : // The implementations here are generic implementations valid for
<span class="lineNum">      33 </span>            : // scalar types (e.g. std::int32_t). Architecture-specific SIMD types
<span class="lineNum">      34 </span>            : // (e.g. NEON int32x4_t) may be supported by providing
<span class="lineNum">      35 </span>            : // specializations for them in separate files.
<span class="lineNum">      36 </span>            : //
<span class="lineNum">      37 </span>            : // The purpose of these primitives is two-fold:
<span class="lineNum">      38 </span>            : //  - They will be used to implement higher-level fixed-point
<span class="lineNum">      39 </span>            : //    abstractions, namely the FixedPoint class and its arithmetic
<span class="lineNum">      40 </span>            : //    operators.
<span class="lineNum">      41 </span>            : //  - They will be directly used to implement some more involved
<span class="lineNum">      42 </span>            : //    fixed-point computations, e.g. the fixed-point implementation
<span class="lineNum">      43 </span>            : //    of math functions such as tanh.
<span class="lineNum">      44 </span>            : 
<span class="lineNum">      45 </span>            : // Some compile-time traits around raw types to handle SIMD aspects:
<span class="lineNum">      46 </span>            : // number of lanes, underlying scalar type.
<span class="lineNum">      47 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">      48 </span>            : struct FixedPointRawTypeTraits {};
<span class="lineNum">      49 </span>            : 
<span class="lineNum">      50 </span>            : template &lt;&gt;
<span class="lineNum">      51 </span>            : struct FixedPointRawTypeTraits&lt;std::int32_t&gt; {
<span class="lineNum">      52 </span>            :   typedef std::int32_t ScalarRawType;
<span class="lineNum">      53 </span>            :   static constexpr int kLanes = 1;
<span class="lineNum">      54 </span>            : };
<span class="lineNum">      55 </span>            : 
<span class="lineNum">      56 </span>            : template &lt;&gt;
<span class="lineNum">      57 </span>            : struct FixedPointRawTypeTraits&lt;std::int16_t&gt; {
<span class="lineNum">      58 </span>            :   typedef std::int16_t ScalarRawType;
<span class="lineNum">      59 </span>            :   static constexpr int kLanes = 1;
<span class="lineNum">      60 </span>            : };
<span class="lineNum">      61 </span>            : 
<a name="62"><span class="lineNum">      62 </span>            : // Returns a SIMD value duplicating a scalar value across all lanes.</a>
<span class="lineNum">      63 </span>            : template &lt;typename tRawType&gt;
<span class="lineNum">      64 </span><span class="lineCov">   33931984 : tRawType Dup(typename FixedPointRawTypeTraits&lt;tRawType&gt;::ScalarRawType x) {</span>
<span class="lineNum">      65 </span><span class="lineCov">   33931984 :   return x;</span>
<span class="lineNum">      66 </span>            : }
<span class="lineNum">      67 </span>            : 
<a name="68"><span class="lineNum">      68 </span>            : // Plain bit-wise AND</a>
<span class="lineNum">      69 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">      70 </span><span class="lineCov">    8278992 : tIntegerType BitAnd(tIntegerType a, tIntegerType b) {</span>
<span class="lineNum">      71 </span><span class="lineCov">    8278992 :   return a &amp; b;</span>
<span class="lineNum">      72 </span>            : }
<span class="lineNum">      73 </span>            : 
<span class="lineNum">      74 </span>            : // Plain bit-wise OR
<span class="lineNum">      75 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">      76 </span>            : tIntegerType BitOr(tIntegerType a, tIntegerType b) {
<span class="lineNum">      77 </span>            :   return a | b;
<span class="lineNum">      78 </span>            : }
<span class="lineNum">      79 </span>            : 
<a name="80"><span class="lineNum">      80 </span>            : // Plain bit-wise XOR</a>
<span class="lineNum">      81 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">      82 </span><span class="lineCov">    1794672 : tIntegerType BitXor(tIntegerType a, tIntegerType b) {</span>
<span class="lineNum">      83 </span><span class="lineCov">    1794672 :   return a ^ b;</span>
<span class="lineNum">      84 </span>            : }
<span class="lineNum">      85 </span>            : 
<a name="86"><span class="lineNum">      86 </span>            : // Plain bit-wise NOT</a>
<span class="lineNum">      87 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">      88 </span><span class="lineCov">    3868602 : tIntegerType BitNot(tIntegerType a) {</span>
<span class="lineNum">      89 </span><span class="lineCov">    3868602 :   return ~a;</span>
<span class="lineNum">      90 </span>            : }
<span class="lineNum">      91 </span>            : 
<a name="92"><span class="lineNum">      92 </span>            : // Integer addition. Not saturating. Overflow is undefined behavior.</a>
<span class="lineNum">      93 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">      94 </span><span class="lineCov">    3513936 : tIntegerType Add(tIntegerType a, tIntegerType b) {</span>
<span class="lineNum">      95 </span><span class="lineCov">    3513936 :   return a + b;</span>
<span class="lineNum">      96 </span>            : }
<span class="lineNum">      97 </span>            : 
<span class="lineNum">      98 </span>            : // Integer multiplication. Not saturating. Overflow is undefined behavior.
<span class="lineNum">      99 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     100 </span>            : tIntegerType Mul(tIntegerType a, tIntegerType b) {
<span class="lineNum">     101 </span>            :   return a * b;
<span class="lineNum">     102 </span>            : }
<span class="lineNum">     103 </span>            : 
<a name="104"><span class="lineNum">     104 </span>            : // Integer subtraction. Not saturating. Overflow is undefined behavior.</a>
<span class="lineNum">     105 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     106 </span><span class="lineCov">     622944 : tIntegerType Sub(tIntegerType a, tIntegerType b) {</span>
<span class="lineNum">     107 </span><span class="lineCov">     622944 :   return a - b;</span>
<span class="lineNum">     108 </span>            : }
<span class="lineNum">     109 </span>            : 
<a name="110"><span class="lineNum">     110 </span>            : // Integer unary negative. Not saturating. Overflow is undefined behavior.</a>
<span class="lineNum">     111 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     112 </span><span class="lineCov">     138432 : tIntegerType Neg(tIntegerType a) {</span>
<span class="lineNum">     113 </span><span class="lineCov">     138432 :   return -a;</span>
<span class="lineNum">     114 </span>            : }
<span class="lineNum">     115 </span>            : 
<span class="lineNum">     116 </span>            : // Integer arithmetic left-shift, equivalent to multiplying with a power of two.
<span class="lineNum">     117 </span>            : // Negative values are OK. In case of overflow, no Undefined
<span class="lineNum">     118 </span>            : // Behavior, but the results are implementation-defined (in practice,
<span class="lineNum">     119 </span>            : // they currently are saturated, but we make no commitment to that). The idea
<span class="lineNum">     120 </span>            : // is that the caller will want to implement the overflowing cases with
<span class="lineNum">     121 </span>            : // saturation with compare-and-mask, so we don't care about the results
<span class="lineNum">     122 </span>            : // in the overflow case, we just want to avoid undefined behavior.
<span class="lineNum">     123 </span>            : //
<a name="124"><span class="lineNum">     124 </span>            : // tIntegerType may be int32 or any narrower signed type.</a>
<span class="lineNum">     125 </span>            : template &lt;typename tIntegerType, typename OffsetType&gt;
<span class="lineNum">     126 </span><span class="lineCov">     467208 : tIntegerType ShiftLeft(tIntegerType a, OffsetType offset) {</span>
<span class="lineNum">     127 </span><span class="lineCov">     467208 :   const std::int64_t wide_a = static_cast&lt;std::int64_t&gt;(a);</span>
<span class="lineNum">     128 </span><span class="lineCov">     467208 :   const std::int64_t wide_shifted = wide_a * (1 &lt;&lt; offset);</span>
<span class="lineNum">     129 </span><span class="lineCov">     467208 :   const auto min = std::numeric_limits&lt;tIntegerType&gt;::min();</span>
<span class="lineNum">     130 </span><span class="lineCov">     467208 :   const auto max = std::numeric_limits&lt;tIntegerType&gt;::max();</span>
<span class="lineNum">     131 </span>            :   return wide_shifted &lt; min
<span class="lineNum">     132 </span>            :              ? min
<span class="lineNum">     133 </span>            :              : wide_shifted &gt; max ? max
<span class="lineNum">     134 </span><span class="lineCov">     467208 :                                   : static_cast&lt;tIntegerType&gt;(wide_shifted);</span>
<span class="lineNum">     135 </span>            : }
<span class="lineNum">     136 </span>            : 
<span class="lineNum">     137 </span>            : // Integer arithmetic right-shift. Not rounding.
<span class="lineNum">     138 </span>            : // Relying on implementation-defined, but in-practice-consistent,
<a name="139"><span class="lineNum">     139 </span>            : // C++ compiler behavior.</a>
<span class="lineNum">     140 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     141 </span><span class="lineCov">    2704544 : tIntegerType ShiftRight(tIntegerType a, int offset) {</span>
<span class="lineNum">     142 </span><span class="lineCov">    2704544 :   return a &gt;&gt; offset;</span>
<span class="lineNum">     143 </span>            : }
<span class="lineNum">     144 </span>            : 
<span class="lineNum">     145 </span>            : // Each bit of the result is set to the corresponding bit of either then_val or
<span class="lineNum">     146 </span>            : // else_val depending on whether the corresponding bit of if_mask is set.
<a name="147"><span class="lineNum">     147 </span>            : // Equivalent to the VBSL instruction in ARM NEON.</a>
<span class="lineNum">     148 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     149 </span><span class="lineCov">    1794672 : tIntegerType SelectUsingMask(tIntegerType if_mask, tIntegerType then_val,</span>
<span class="lineNum">     150 </span>            :                              tIntegerType else_val) {
<span class="lineNum">     151 </span><span class="lineCov">    1794672 :   return BitXor(BitAnd(if_mask, then_val), BitAnd(BitNot(if_mask), else_val));</span>
<span class="lineNum">     152 </span>            : }
<span class="lineNum">     153 </span>            : 
<span class="lineNum">     154 </span>            : // For each input scalar, the corresponding bits of the result are set if the
<a name="155"><span class="lineNum">     155 </span>            : // input scalar is non-zero.</a>
<span class="lineNum">     156 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     157 </span><span class="lineCov">    5408912 : tIntegerType MaskIfNonZero(tIntegerType a) {</span>
<span class="lineNum">     158 </span>            :   static constexpr tIntegerType zero = 0;
<span class="lineNum">     159 </span><span class="lineCov">    5408912 :   return a ? BitNot(zero) : zero;</span>
<span class="lineNum">     160 </span>            : }
<span class="lineNum">     161 </span>            : 
<span class="lineNum">     162 </span>            : // For each input scalar, the corresponding bits of the result are set if the
<a name="163"><span class="lineNum">     163 </span>            : // input scalar is zero.</a>
<span class="lineNum">     164 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     165 </span><span class="lineCov">     173040 : tIntegerType MaskIfZero(tIntegerType a) {</span>
<span class="lineNum">     166 </span><span class="lineCov">     173040 :   return MaskIfNonZero&lt;tIntegerType&gt;(!a);</span>
<span class="lineNum">     167 </span>            : }
<span class="lineNum">     168 </span>            : 
<span class="lineNum">     169 </span>            : // For each pair of input scalars, the corresponding bits of the result are
<a name="170"><span class="lineNum">     170 </span>            : // set if the input scalars are equal.</a>
<span class="lineNum">     171 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     172 </span><span class="lineCov">     978912 : tIntegerType MaskIfEqual(tIntegerType a, tIntegerType b) {</span>
<span class="lineNum">     173 </span><span class="lineCov">     978912 :   return MaskIfNonZero&lt;tIntegerType&gt;(a == b);</span>
<span class="lineNum">     174 </span>            : }
<span class="lineNum">     175 </span>            : 
<span class="lineNum">     176 </span>            : // For each pair of input scalars, the corresponding bits of the result are
<span class="lineNum">     177 </span>            : // set if the input scalars are not equal.
<span class="lineNum">     178 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     179 </span>            : tIntegerType MaskIfNotEqual(tIntegerType a, tIntegerType b) {
<span class="lineNum">     180 </span>            :   return MaskIfNonZero&lt;tIntegerType&gt;(a != b);
<span class="lineNum">     181 </span>            : }
<span class="lineNum">     182 </span>            : 
<span class="lineNum">     183 </span>            : // For each pair of input scalars, the corresponding bits of the result are
<a name="184"><span class="lineNum">     184 </span>            : // set if the input scalars a, b satisfy a &gt; b.</a>
<span class="lineNum">     185 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     186 </span><span class="lineCov">    1854088 : tIntegerType MaskIfGreaterThan(tIntegerType a, tIntegerType b) {</span>
<span class="lineNum">     187 </span><span class="lineCov">    1854088 :   return MaskIfNonZero&lt;tIntegerType&gt;(a &gt; b);</span>
<span class="lineNum">     188 </span>            : }
<span class="lineNum">     189 </span>            : 
<span class="lineNum">     190 </span>            : // For each pair of input scalars, the corresponding bits of the result are
<span class="lineNum">     191 </span>            : // set if the input scalars a, b satisfy a &gt;= b.
<span class="lineNum">     192 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     193 </span>            : tIntegerType MaskIfGreaterThanOrEqual(tIntegerType a, tIntegerType b) {
<span class="lineNum">     194 </span>            :   return MaskIfNonZero&lt;tIntegerType&gt;(a &gt;= b);
<span class="lineNum">     195 </span>            : }
<span class="lineNum">     196 </span>            : 
<span class="lineNum">     197 </span>            : // For each pair of input scalars, the corresponding bits of the result are
<a name="198"><span class="lineNum">     198 </span>            : // set if the input scalars a, b satisfy a &lt; b.</a>
<span class="lineNum">     199 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     200 </span><span class="lineCov">    1873864 : tIntegerType MaskIfLessThan(tIntegerType a, tIntegerType b) {</span>
<span class="lineNum">     201 </span><span class="lineCov">    1873864 :   return MaskIfNonZero&lt;tIntegerType&gt;(a &lt; b);</span>
<span class="lineNum">     202 </span>            : }
<span class="lineNum">     203 </span>            : 
<span class="lineNum">     204 </span>            : // For each pair of input scalars, the corresponding bits of the result are
<span class="lineNum">     205 </span>            : // set if the input scalars a, b satisfy a &lt;= b.
<span class="lineNum">     206 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     207 </span>            : tIntegerType MaskIfLessThanOrEqual(tIntegerType a, tIntegerType b) {
<span class="lineNum">     208 </span>            :   return MaskIfNonZero&lt;tIntegerType&gt;(a &lt;= b);
<span class="lineNum">     209 </span>            : }
<span class="lineNum">     210 </span>            : 
<span class="lineNum">     211 </span>            : // Returns true if all of the input scalars are nonzero.
<span class="lineNum">     212 </span>            : // This function may currently assume that each of the input scalars has either
<a name="213"><span class="lineNum">     213 </span>            : // all or none of its bits set. Otherwise, its behavior is currently undefined.</a>
<span class="lineNum">     214 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     215 </span><span class="lineCov">     978912 : bool All(tIntegerType a) {</span>
<span class="lineNum">     216 </span><span class="lineCov">     978912 :   return a;</span>
<span class="lineNum">     217 </span>            : }
<span class="lineNum">     218 </span>            : 
<span class="lineNum">     219 </span>            : // Returns true if any of the input scalars are nonzero.
<span class="lineNum">     220 </span>            : // This function may currently assume that each of the input scalars has either
<span class="lineNum">     221 </span>            : // all or none of its bits set. Otherwise, its behavior is currently undefined.
<span class="lineNum">     222 </span>            : template &lt;typename tIntegerType&gt;
<span class="lineNum">     223 </span>            : bool Any(tIntegerType a) {
<span class="lineNum">     224 </span>            :   return a;
<span class="lineNum">     225 </span>            : }
<span class="lineNum">     226 </span>            : 
<span class="lineNum">     227 </span>            : // Returns (a+b)/2, rounded to the nearest integer.
<span class="lineNum">     228 </span>            : // Equivalent to VRHADD in the ARM NEON instruction set.
<span class="lineNum">     229 </span>            : template &lt;typename IntegerType&gt;
<span class="lineNum">     230 </span>            : IntegerType RoundingHalfSum(IntegerType a, IntegerType b) {
<span class="lineNum">     231 </span>            :   static_assert(std::is_same&lt;IntegerType, void&gt;::value, &quot;unimplemented&quot;);
<span class="lineNum">     232 </span>            :   (void)b;
<span class="lineNum">     233 </span>            :   return a;
<span class="lineNum">     234 </span>            : }
<a name="235"><span class="lineNum">     235 </span>            : </a>
<span class="lineNum">     236 </span>            : template &lt;&gt;
<span class="lineNum">     237 </span><span class="lineCov">      42112 : inline std::int32_t RoundingHalfSum(std::int32_t a, std::int32_t b) {</span>
<span class="lineNum">     238 </span><span class="lineCov">      42112 :   std::int64_t a64 = a;</span>
<span class="lineNum">     239 </span><span class="lineCov">      42112 :   std::int64_t b64 = b;</span>
<span class="lineNum">     240 </span><span class="lineCov">      42112 :   std::int64_t sum = a64 + b64;</span>
<span class="lineNum">     241 </span><span class="lineCov">      42112 :   std::int64_t sign = sum &gt;= 0 ? 1 : -1;</span>
<span class="lineNum">     242 </span><span class="lineCov">      42112 :   return static_cast&lt;std::int32_t&gt;((sum + sign) / 2);</span>
<span class="lineNum">     243 </span>            : }
<a name="244"><span class="lineNum">     244 </span>            : </a>
<span class="lineNum">     245 </span>            : template &lt;&gt;
<span class="lineNum">     246 </span><span class="lineCov">      36992 : inline std::int16_t RoundingHalfSum(std::int16_t a, std::int16_t b) {</span>
<span class="lineNum">     247 </span><span class="lineCov">      36992 :   std::int32_t a32 = a;</span>
<span class="lineNum">     248 </span><span class="lineCov">      36992 :   std::int32_t b32 = b;</span>
<span class="lineNum">     249 </span><span class="lineCov">      36992 :   std::int32_t sum = a32 + b32;</span>
<span class="lineNum">     250 </span><span class="lineCov">      36992 :   std::int32_t sign = sum &gt;= 0 ? 1 : -1;</span>
<span class="lineNum">     251 </span><span class="lineCov">      36992 :   return static_cast&lt;std::int16_t&gt;((sum + sign) / 2);</span>
<span class="lineNum">     252 </span>            : }
<span class="lineNum">     253 </span>            : 
<span class="lineNum">     254 </span>            : template &lt;typename IntegerType&gt;
<span class="lineNum">     255 </span>            : IntegerType SaturatingAdd(IntegerType a, IntegerType b) {
<span class="lineNum">     256 </span>            :   static_assert(std::is_same&lt;IntegerType, void&gt;::value, &quot;unimplemented&quot;);
<span class="lineNum">     257 </span>            :   (void)b;
<span class="lineNum">     258 </span>            :   return a;
<span class="lineNum">     259 </span>            : }
<span class="lineNum">     260 </span>            : 
<a name="261"><span class="lineNum">     261 </span>            : // So far this is only needed for int16.</a>
<span class="lineNum">     262 </span>            : template &lt;&gt;
<span class="lineNum">     263 </span><span class="lineCov">      50864 : inline std::int16_t SaturatingAdd(std::int16_t a, std::int16_t b) {</span>
<span class="lineNum">     264 </span><span class="lineCov">      50864 :   std::int32_t a32 = a;</span>
<span class="lineNum">     265 </span><span class="lineCov">      50864 :   std::int32_t b32 = b;</span>
<span class="lineNum">     266 </span><span class="lineCov">      50864 :   std::int32_t sum = a32 + b32;</span>
<span class="lineNum">     267 </span>            :   return static_cast&lt;std::int16_t&gt;(
<span class="lineNum">     268 </span><span class="lineCov">      50864 :       std::min(static_cast&lt;std::int32_t&gt;(32767),</span>
<span class="lineNum">     269 </span><span class="lineCov">     101728 :                std::max(static_cast&lt;std::int32_t&gt;(-32768), sum)));</span>
<span class="lineNum">     270 </span>            : }
<span class="lineNum">     271 </span>            : 
<span class="lineNum">     272 </span>            : template &lt;&gt;
<span class="lineNum">     273 </span>            : inline std::int8_t SaturatingAdd(std::int8_t a, std::int8_t b) {
<span class="lineNum">     274 </span>            :   std::int16_t a16 = a;
<span class="lineNum">     275 </span>            :   std::int16_t b16 = b;
<span class="lineNum">     276 </span>            :   std::int16_t sum = a16 + b16;
<span class="lineNum">     277 </span>            :   return static_cast&lt;std::int8_t&gt;(std::min(
<span class="lineNum">     278 </span>            :       static_cast&lt;int16_t&gt;(std::numeric_limits&lt;int8_t&gt;::max()),
<span class="lineNum">     279 </span>            :       std::max(static_cast&lt;int16_t&gt;(std::numeric_limits&lt;int8_t&gt;::min()), sum)));
<span class="lineNum">     280 </span>            : }
<span class="lineNum">     281 </span>            : 
<span class="lineNum">     282 </span>            : // Returns a+b, saturating if the integers are 16bit or narrower,
<span class="lineNum">     283 </span>            : // otherwise just a plain addition.
<a name="284"><span class="lineNum">     284 </span>            : template &lt;typename IntegerType, bool Is16Bit&gt;</a>
<span class="lineNum">     285 </span>            : struct AddSaturatingIf16BitImpl {
<span class="lineNum">     286 </span><span class="lineCov">      57904 :   static IntegerType Run(IntegerType a, IntegerType b) { return Add(a, b); }</span>
<span class="lineNum">     287 </span>            : };
<a name="288"><span class="lineNum">     288 </span>            : template &lt;typename IntegerType&gt;</a>
<span class="lineNum">     289 </span>            : struct AddSaturatingIf16BitImpl&lt;IntegerType, true&gt; {
<span class="lineNum">     290 </span><span class="lineCov">      50864 :   static IntegerType Run(IntegerType a, IntegerType b) {</span>
<span class="lineNum">     291 </span><span class="lineCov">      50864 :     return SaturatingAdd(a, b);</span>
<span class="lineNum">     292 </span>            :   }
<a name="293"><span class="lineNum">     293 </span>            : };</a>
<span class="lineNum">     294 </span>            : template &lt;typename IntegerType&gt;
<span class="lineNum">     295 </span><span class="lineCov">     108768 : IntegerType AddSaturatingIf16Bit(IntegerType a, IntegerType b) {</span>
<span class="lineNum">     296 </span>            :   using ScalarType =
<span class="lineNum">     297 </span>            :       typename FixedPointRawTypeTraits&lt;IntegerType&gt;::ScalarRawType;
<span class="lineNum">     298 </span><span class="lineCov">      50864 :   return AddSaturatingIf16BitImpl&lt;IntegerType, sizeof(ScalarType) == 2&gt;::Run(a,</span>
<span class="lineNum">     299 </span><span class="lineCov">     108768 :                                                                              b);</span>
<span class="lineNum">     300 </span>            : }
<span class="lineNum">     301 </span>            : 
<span class="lineNum">     302 </span>            : // Returns the integer that represents the product of two fixed-point
<span class="lineNum">     303 </span>            : // numbers, interpreting all integers as fixed-point values in the
<span class="lineNum">     304 </span>            : // interval [-1, 1), rounding to the nearest value, and saturating
<span class="lineNum">     305 </span>            : // -1 * -1 to the maximum value (since 1 is not in the half-open
<span class="lineNum">     306 </span>            : // interval [-1, 1)).
<span class="lineNum">     307 </span>            : //
<span class="lineNum">     308 </span>            : // [The explanation below specializes to std::int32_t for example purpose.]
<span class="lineNum">     309 </span>            : //
<span class="lineNum">     310 </span>            : // The mapping between IntegerType and the interval [-1, 1) is unique and
<span class="lineNum">     311 </span>            : // implied by IntegerType, which is assumed to be signed. For example,
<span class="lineNum">     312 </span>            : // for IntegerType==std::int32_t, the mapping is
<span class="lineNum">     313 </span>            : //   real_value = integer_value / 2^31.
<span class="lineNum">     314 </span>            : // So in this case, and leaving aside rounding and saturating, this
<span class="lineNum">     315 </span>            : // function computes ((a / 2^31) * (b / 2^31)) * 2^31, which simplifies to
<span class="lineNum">     316 </span>            : //   (a * b) / 2^31.
<span class="lineNum">     317 </span>            : //
<span class="lineNum">     318 </span>            : // The 'doubling' part in the name of this function comes from the fact that
<span class="lineNum">     319 </span>            : // this operation is very close to a &quot;multiply-high&quot; operation, keeping only
<span class="lineNum">     320 </span>            : // the top half bits, except that that would be effectively computing
<span class="lineNum">     321 </span>            : //   (a * b) / 2^32,
<span class="lineNum">     322 </span>            : // so here we are computing 2x that, since
<span class="lineNum">     323 </span>            : //   1/2^31 = 2 * 1/2^32.
<span class="lineNum">     324 </span>            : // The idea is to use all of the available 32 bits in the destination int32
<span class="lineNum">     325 </span>            : // value.
<span class="lineNum">     326 </span>            : //
<span class="lineNum">     327 </span>            : // [End of the explanation specializing to int32.]
<span class="lineNum">     328 </span>            : //
<span class="lineNum">     329 </span>            : // This is equivalent to the VQRDMULH instruction in ARM NEON.
<span class="lineNum">     330 </span>            : template &lt;typename IntegerType&gt;
<span class="lineNum">     331 </span>            : IntegerType SaturatingRoundingDoublingHighMul(IntegerType a, IntegerType b) {
<span class="lineNum">     332 </span>            :   static_assert(std::is_same&lt;IntegerType, void&gt;::value, &quot;unimplemented&quot;);
<span class="lineNum">     333 </span>            :   (void)b;
<span class="lineNum">     334 </span>            :   return a;
<span class="lineNum">     335 </span>            : }
<span class="lineNum">     336 </span>            : 
<span class="lineNum">     337 </span>            : // This function implements the same computation as the ARMv7 NEON VQRDMULH
<a name="338"><span class="lineNum">     338 </span>            : // instruction.</a>
<span class="lineNum">     339 </span>            : template &lt;&gt;
<span class="lineNum">     340 </span><span class="lineCov">   14720776 : inline std::int32_t SaturatingRoundingDoublingHighMul(std::int32_t a,</span>
<span class="lineNum">     341 </span>            :                                                       std::int32_t b) {
<span class="lineNum">     342 </span><span class="lineCov">   14720776 :   bool overflow = a == b &amp;&amp; a == std::numeric_limits&lt;std::int32_t&gt;::min();</span>
<span class="lineNum">     343 </span><span class="lineCov">   14720776 :   std::int64_t a_64(a);</span>
<span class="lineNum">     344 </span><span class="lineCov">   14720776 :   std::int64_t b_64(b);</span>
<span class="lineNum">     345 </span><span class="lineCov">   14720776 :   std::int64_t ab_64 = a_64 * b_64;</span>
<span class="lineNum">     346 </span><span class="lineCov">   14720776 :   std::int32_t nudge = ab_64 &gt;= 0 ? (1 &lt;&lt; 30) : (1 - (1 &lt;&lt; 30));</span>
<span class="lineNum">     347 </span>            :   std::int32_t ab_x2_high32 =
<span class="lineNum">     348 </span><span class="lineCov">   14720776 :       static_cast&lt;std::int32_t&gt;((ab_64 + nudge) / (1ll &lt;&lt; 31));</span>
<span class="lineNum">     349 </span><span class="lineCov">   14720776 :   return overflow ? std::numeric_limits&lt;std::int32_t&gt;::max() : ab_x2_high32;</span>
<span class="lineNum">     350 </span>            : }
<a name="351"><span class="lineNum">     351 </span>            : </a>
<span class="lineNum">     352 </span>            : template &lt;&gt;
<span class="lineNum">     353 </span><span class="lineCov">   11451336 : inline std::int16_t SaturatingRoundingDoublingHighMul(std::int16_t a,</span>
<span class="lineNum">     354 </span>            :                                                       std::int16_t b) {
<span class="lineNum">     355 </span><span class="lineCov">   11451336 :   bool overflow = a == b &amp;&amp; a == std::numeric_limits&lt;std::int16_t&gt;::min();</span>
<span class="lineNum">     356 </span><span class="lineCov">   11451336 :   std::int32_t a_32(a);</span>
<span class="lineNum">     357 </span><span class="lineCov">   11451336 :   std::int32_t b_32(b);</span>
<span class="lineNum">     358 </span><span class="lineCov">   11451336 :   std::int32_t ab_32 = a_32 * b_32;</span>
<span class="lineNum">     359 </span><span class="lineCov">   11451336 :   std::int16_t nudge = ab_32 &gt;= 0 ? (1 &lt;&lt; 14) : (1 - (1 &lt;&lt; 14));</span>
<span class="lineNum">     360 </span>            :   std::int16_t ab_x2_high16 =
<span class="lineNum">     361 </span><span class="lineCov">   11451336 :       static_cast&lt;std::int16_t&gt;((ab_32 + nudge) / (1 &lt;&lt; 15));</span>
<span class="lineNum">     362 </span><span class="lineCov">   11451336 :   return overflow ? std::numeric_limits&lt;std::int16_t&gt;::max() : ab_x2_high16;</span>
<span class="lineNum">     363 </span>            : }
<span class="lineNum">     364 </span>            : 
<span class="lineNum">     365 </span>            : // Correctly-rounded-to-nearest division by a power-of-two.
<a name="366"><span class="lineNum">     366 </span>            : // Also known as a rounding arithmetic right shift.</a>
<span class="lineNum">     367 </span>            : template &lt;typename IntegerType, typename ExponentType&gt;
<span class="lineNum">     368 </span><span class="lineCov">    1352272 : inline IntegerType RoundingDivideByPOT(IntegerType x, ExponentType exponent) {</span>
<span class="lineNum">     369 </span><span class="lineCov">    1352272 :   assert(exponent &gt;= 0);</span>
<span class="lineNum">     370 </span><span class="lineCov">    1352272 :   assert(exponent &lt;= 31);</span>
<span class="lineNum">     371 </span><span class="lineCov">    1352272 :   const IntegerType mask = Dup&lt;IntegerType&gt;((1ll &lt;&lt; exponent) - 1);</span>
<span class="lineNum">     372 </span><span class="lineCov">    1352272 :   const IntegerType zero = Dup&lt;IntegerType&gt;(0);</span>
<span class="lineNum">     373 </span><span class="lineCov">    1352272 :   const IntegerType one = Dup&lt;IntegerType&gt;(1);</span>
<span class="lineNum">     374 </span><span class="lineCov">    1352272 :   const IntegerType remainder = BitAnd(x, mask);</span>
<span class="lineNum">     375 </span>            :   const IntegerType threshold =
<span class="lineNum">     376 </span><span class="lineCov">    1352272 :       Add(ShiftRight(mask, 1), BitAnd(MaskIfLessThan(x, zero), one));</span>
<span class="lineNum">     377 </span><span class="lineCov">    1352272 :   return Add(ShiftRight(x, exponent),</span>
<span class="lineNum">     378 </span><span class="lineCov">    1516424 :              BitAnd(MaskIfGreaterThan(remainder, threshold), one));</span>
<span class="lineNum">     379 </span>            : }
<span class="lineNum">     380 </span>            : 
<span class="lineNum">     381 </span>            : // Returns the product of a run-time integer value by a compile-time power
<span class="lineNum">     382 </span>            : // of two, with either a positive exponent (equivalent to an arithmetic
<span class="lineNum">     383 </span>            : // left shift, saturating) or a negative exponent (equivalent to an arithmetic
<span class="lineNum">     384 </span>            : // right shift, rounding to nearest).
<span class="lineNum">     385 </span>            : template &lt;int Exponent, typename IntegerType,
<span class="lineNum">     386 </span>            :           int ExponentSign = (Exponent &gt; 0 ? 1 : Exponent &lt; 0 ? -1 : 0)&gt;
<span class="lineNum">     387 </span>            : struct ImplSaturatingRoundingMultiplyByPOT {};
<span class="lineNum">     388 </span>            : 
<a name="389"><span class="lineNum">     389 </span>            : template &lt;int Exponent, typename IntegerType&gt;</a>
<span class="lineNum">     390 </span>            : struct ImplSaturatingRoundingMultiplyByPOT&lt;Exponent, IntegerType, 0&gt; {
<span class="lineNum">     391 </span><span class="lineCov">      19776 :   static IntegerType eval(IntegerType x) { return x; }</span>
<span class="lineNum">     392 </span>            : };
<span class="lineNum">     393 </span>            : 
<a name="394"><span class="lineNum">     394 </span>            : template &lt;int Exponent, typename IntegerType&gt;</a>
<span class="lineNum">     395 </span>            : struct ImplSaturatingRoundingMultiplyByPOT&lt;Exponent, IntegerType, 1&gt; {
<span class="lineNum">     396 </span><span class="lineCov">     467208 :   static IntegerType eval(IntegerType x) {</span>
<span class="lineNum">     397 </span>            :     using ScalarIntegerType =
<span class="lineNum">     398 </span>            :         typename FixedPointRawTypeTraits&lt;IntegerType&gt;::ScalarRawType;
<span class="lineNum">     399 </span>            :     const IntegerType min =
<span class="lineNum">     400 </span><span class="lineCov">     467208 :         Dup&lt;IntegerType&gt;(std::numeric_limits&lt;ScalarIntegerType&gt;::min());</span>
<span class="lineNum">     401 </span>            :     const IntegerType max =
<span class="lineNum">     402 </span><span class="lineCov">     467208 :         Dup&lt;IntegerType&gt;(std::numeric_limits&lt;ScalarIntegerType&gt;::max());</span>
<span class="lineNum">     403 </span><span class="lineCov">     467208 :     const int ScalarIntegerTypeBits = 8 * sizeof(ScalarIntegerType);</span>
<span class="lineNum">     404 </span>            : 
<span class="lineNum">     405 </span>            :     const std::int32_t threshold =
<span class="lineNum">     406 </span><span class="lineCov">     467208 :         ((1 &lt;&lt; (ScalarIntegerTypeBits - 1 - Exponent)) - 1);</span>
<span class="lineNum">     407 </span>            :     const IntegerType positive_mask =
<span class="lineNum">     408 </span><span class="lineCov">     467208 :         MaskIfGreaterThan(x, Dup&lt;IntegerType&gt;(threshold));</span>
<span class="lineNum">     409 </span>            :     const IntegerType negative_mask =
<span class="lineNum">     410 </span><span class="lineCov">     467208 :         MaskIfLessThan(x, Dup&lt;IntegerType&gt;(-threshold));</span>
<span class="lineNum">     411 </span>            : 
<span class="lineNum">     412 </span><span class="lineCov">     467208 :     IntegerType result = ShiftLeft(x, Exponent);</span>
<span class="lineNum">     413 </span><span class="lineCov">     467208 :     result = SelectUsingMask(positive_mask, max, result);</span>
<span class="lineNum">     414 </span><span class="lineCov">     467208 :     result = SelectUsingMask(negative_mask, min, result);</span>
<span class="lineNum">     415 </span><span class="lineCov">     467208 :     return result;</span>
<span class="lineNum">     416 </span>            :   }
<span class="lineNum">     417 </span>            : };
<span class="lineNum">     418 </span>            : 
<a name="419"><span class="lineNum">     419 </span>            : template &lt;int Exponent, typename IntegerType&gt;</a>
<span class="lineNum">     420 </span>            : struct ImplSaturatingRoundingMultiplyByPOT&lt;Exponent, IntegerType, -1&gt; {
<span class="lineNum">     421 </span><span class="lineCov">     271920 :   static IntegerType eval(IntegerType x) {</span>
<span class="lineNum">     422 </span><span class="lineCov">     271920 :     return RoundingDivideByPOT&lt;IntegerType&gt;(x, -Exponent);</span>
<span class="lineNum">     423 </span>            :   }
<span class="lineNum">     424 </span>            : };
<a name="425"><span class="lineNum">     425 </span>            : </a>
<span class="lineNum">     426 </span>            : template &lt;int Exponent, typename IntegerType&gt;
<span class="lineNum">     427 </span><span class="lineCov">     758904 : IntegerType SaturatingRoundingMultiplyByPOT(IntegerType x) {</span>
<span class="lineNum">     428 </span><span class="lineCov">     758904 :   return ImplSaturatingRoundingMultiplyByPOT&lt;Exponent, IntegerType&gt;::eval(x);</span>
<span class="lineNum">     429 </span>            : }
<span class="lineNum">     430 </span>            : 
<span class="lineNum">     431 </span>            : // Part 2: the FixedPoint class.
<span class="lineNum">     432 </span>            : 
<span class="lineNum">     433 </span>            : // A FixedPoint object represents a fixed-point value stored in the underlying
<span class="lineNum">     434 </span>            : // integer type tRawType, if tRawType is a plain scalar integer type.
<span class="lineNum">     435 </span>            : // Alternatively, tRawType may be a SIMD type (e.g. NEON int32x4_t) in which
<span class="lineNum">     436 </span>            : // case a FixedPoint object represents a corresponding SIMD vector of fixed
<span class="lineNum">     437 </span>            : // point values.
<span class="lineNum">     438 </span>            : //
<span class="lineNum">     439 </span>            : // tIntegerBits describes the range of the fixed-point format: if
<span class="lineNum">     440 </span>            : // tIntegerBits == m then the range of representable values is the half-open
<span class="lineNum">     441 </span>            : // interval [-2^m; 2^m) where the open boundary on the right side means that
<span class="lineNum">     442 </span>            : // 2^m is not representable (how close the maximum representable value is to
<span class="lineNum">     443 </span>            : // it, depends on bit-depth of tRawType).
<span class="lineNum">     444 </span>            : //
<span class="lineNum">     445 </span>            : // In &quot;Q format notation&quot;,
<span class="lineNum">     446 </span>            : //   https://en.wikipedia.org/wiki/Q_(number_format)
<span class="lineNum">     447 </span>            : // we are describing the format
<span class="lineNum">     448 </span>            : //   Qm.n
<span class="lineNum">     449 </span>            : // where
<span class="lineNum">     450 </span>            : //   m = tIntegerBits
<span class="lineNum">     451 </span>            : // and
<span class="lineNum">     452 </span>            : //   n = NumberOfBits(tRawType) - (m + 1)
<span class="lineNum">     453 </span>            : // Note that the (m + 1) in the above line is because we adopt the convention
<span class="lineNum">     454 </span>            : // that we count the integer bits exclusively of the sign bit; so (m + 1) is
<span class="lineNum">     455 </span>            : // the total number of integer bits inclusive of the sign bit.
<span class="lineNum">     456 </span>            : //
<span class="lineNum">     457 </span>            : // Accordingly, the number of integral representable values in our range
<span class="lineNum">     458 </span>            : //   [-2^m ; 2^m)
<span class="lineNum">     459 </span>            : // is equal to 2^(m+1).
<span class="lineNum">     460 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     461 </span>            : class FixedPoint {
<span class="lineNum">     462 </span>            :  public:
<span class="lineNum">     463 </span>            :   typedef tRawType RawType;
<span class="lineNum">     464 </span>            : 
<span class="lineNum">     465 </span>            :   typedef FixedPointRawTypeTraits&lt;RawType&gt; RawTypeTraits;
<span class="lineNum">     466 </span>            :   typedef typename RawTypeTraits::ScalarRawType ScalarRawType;
<span class="lineNum">     467 </span>            : 
<span class="lineNum">     468 </span>            :   static constexpr int kTotalBits = 8 * sizeof(ScalarRawType);
<span class="lineNum">     469 </span>            :   static constexpr int kIntegerBits = tIntegerBits;
<span class="lineNum">     470 </span>            :   static constexpr int kFractionalBits = kTotalBits - 1 - kIntegerBits;
<span class="lineNum">     471 </span>            :   static_assert(kIntegerBits &gt;= 0 &amp;&amp; kIntegerBits &lt; kTotalBits,
<span class="lineNum">     472 </span>            :                 &quot;bad IntegerBits&quot;);
<span class="lineNum">     473 </span>            : 
<a name="474"><span class="lineNum">     474 </span>            :   typedef FixedPoint&lt;ScalarRawType, kIntegerBits&gt; ScalarFixedPointType;</a>
<span class="lineNum">     475 </span>            : 
<span class="lineNum">     476 </span><span class="lineCov">   25583776 :   static const ScalarRawType ScalarRawMin() {</span>
<span class="lineNum">     477 </span><span class="lineCov">   25583776 :     return std::numeric_limits&lt;ScalarRawType&gt;::min();</span>
<a name="478"><span class="lineNum">     478 </span>            :   }</a>
<span class="lineNum">     479 </span>            : 
<span class="lineNum">     480 </span><span class="lineCov">   25801312 :   static const ScalarRawType ScalarRawMax() {</span>
<span class="lineNum">     481 </span><span class="lineCov">   25801312 :     return std::numeric_limits&lt;ScalarRawType&gt;::max();</span>
<span class="lineNum">     482 </span>            :   }
<span class="lineNum">     483 </span>            : 
<span class="lineNum">     484 </span>            :   static const ScalarRawType RawMin() {
<span class="lineNum">     485 </span>            :     return VectorFromScalar(ScalarRawMin());
<span class="lineNum">     486 </span>            :   }
<span class="lineNum">     487 </span>            : 
<span class="lineNum">     488 </span>            :   static const ScalarRawType RawMax() {
<span class="lineNum">     489 </span>            :     return VectorFromScalar(ScalarRawMax());
<a name="490"><span class="lineNum">     490 </span>            :   }</a>
<span class="lineNum">     491 </span>            : 
<span class="lineNum">     492 </span><span class="lineCov">    3060336 :   static FixedPoint FromRaw(RawType x) {</span>
<span class="lineNum">     493 </span>            :     FixedPoint retval;
<span class="lineNum">     494 </span><span class="lineCov">    3060336 :     retval.raw() = x;</span>
<span class="lineNum">     495 </span><span class="lineCov">    3060336 :     return retval;</span>
<a name="496"><span class="lineNum">     496 </span>            :   }</a>
<span class="lineNum">     497 </span>            : 
<span class="lineNum">     498 </span><span class="lineCov">   27477328 :   static FixedPoint FromScalarRaw(ScalarRawType x) {</span>
<span class="lineNum">     499 </span>            :     FixedPoint retval;
<span class="lineNum">     500 </span><span class="lineCov">   27477328 :     retval.raw() = Dup&lt;RawType&gt;(x);</span>
<span class="lineNum">     501 </span><span class="lineCov">   27477328 :     return retval;</span>
<span class="lineNum">     502 </span>            :   }
<span class="lineNum">     503 </span>            : 
<span class="lineNum">     504 </span>            :   static FixedPoint FromScalarFixedPoint(ScalarFixedPointType x) {
<span class="lineNum">     505 </span>            :     return FromScalarRaw(x.raw());
<span class="lineNum">     506 </span>            :   }
<a name="507"><span class="lineNum">     507 </span>            : </a>
<span class="lineNum">     508 </span>            :   template &lt;int Exponent&gt;
<span class="lineNum">     509 </span><span class="lineCov">     212592 :   static FixedPoint ConstantPOT() {</span>
<span class="lineNum">     510 </span>            :     static constexpr int kOffset = kFractionalBits + Exponent;
<span class="lineNum">     511 </span>            :     static_assert(
<span class="lineNum">     512 </span>            :         kOffset &lt; 31,
<span class="lineNum">     513 </span>            :         &quot;Constant not exactly representable in this fixed-point format&quot;);
<span class="lineNum">     514 </span><span class="lineCov">     212592 :     return FromScalarRaw(ScalarRawType(1) &lt;&lt; kOffset);</span>
<a name="515"><span class="lineNum">     515 </span>            :   }</a>
<span class="lineNum">     516 </span>            : 
<a name="517"><span class="lineNum">     517 </span><span class="lineCov">     123600 :   static FixedPoint Zero() { return FromScalarRaw(0); }</span></a>
<span class="lineNum">     518 </span>            : 
<span class="lineNum">     519 </span><span class="lineCov">     494400 :   static FixedPoint One() {</span>
<span class="lineNum">     520 </span><span class="lineCov">     217536 :     return FromScalarRaw(</span>
<span class="lineNum">     521 </span>            :         kIntegerBits == 0
<span class="lineNum">     522 </span><span class="lineCov">     101728 :             ? ScalarRawMax()</span>
<span class="lineNum">     523 </span><span class="lineCov">     494400 :             : (ScalarRawType(1) &lt;&lt; (kIntegerBits == 0 ? 0 : kFractionalBits)));</span>
<a name="524"><span class="lineNum">     524 </span>            :   }</a>
<span class="lineNum">     525 </span>            : 
<span class="lineNum">     526 </span><span class="lineCov">   25583776 :   static FixedPoint FromDouble(double x) {</span>
<span class="lineNum">     527 </span><span class="lineCov">   25583776 :     const double min_bound = static_cast&lt;double&gt;(ScalarRawMin());</span>
<span class="lineNum">     528 </span><span class="lineCov">   25583776 :     const double max_bound = static_cast&lt;double&gt;(ScalarRawMax());</span>
<span class="lineNum">     529 </span><span class="lineCov">   25583776 :     return FromScalarRaw(static_cast&lt;ScalarRawType&gt;(std::min(</span>
<span class="lineNum">     530 </span><span class="lineCov">   25583776 :         std::max(round(x * static_cast&lt;double&gt;(1ll &lt;&lt; kFractionalBits)),</span>
<span class="lineNum">     531 </span><span class="lineCov">   51167552 :                  min_bound),</span>
<span class="lineNum">     532 </span><span class="lineCov">   76751328 :         max_bound)));</span>
<a name="533"><span class="lineNum">     533 </span>            :   }</a>
<a name="534"><span class="lineNum">     534 </span>            : </a>
<span class="lineNum">     535 </span><span class="lineCov">       4944 :   RawType raw() const { return i_; }</span>
<span class="lineNum">     536 </span><span class="lineCov">  265518112 :   RawType&amp; raw() { return i_; }</span>
<span class="lineNum">     537 </span>            : 
<span class="lineNum">     538 </span>            :  private:
<span class="lineNum">     539 </span>            :   RawType i_;
<span class="lineNum">     540 </span>            : };
<span class="lineNum">     541 </span>            : 
<span class="lineNum">     542 </span>            : // Part 3: implementation of arithmetic operators for the
<span class="lineNum">     543 </span>            : // FixedPoint class, and a few related functions.
<span class="lineNum">     544 </span>            : 
<span class="lineNum">     545 </span>            : // A FixedPoint multiplication is just a
<span class="lineNum">     546 </span>            : // SaturatingRoundingDoublingHighMul operation on the underlying
<span class="lineNum">     547 </span>            : // raw integer values. The IntegerBits simply add up, as is obvious
<a name="548"><span class="lineNum">     548 </span>            : // from the fact that the range is [-2^IntegerBits, 2^IntegerBits).</a>
<span class="lineNum">     549 </span>            : template &lt;typename tRawType, int tIntegerBits_a, int tIntegerBits_b&gt;
<span class="lineNum">     550 </span><span class="lineCov">   26172112 : FixedPoint&lt;tRawType, tIntegerBits_a + tIntegerBits_b&gt; operator*(</span>
<span class="lineNum">     551 </span>            :     FixedPoint&lt;tRawType, tIntegerBits_a&gt; a,
<span class="lineNum">     552 </span>            :     FixedPoint&lt;tRawType, tIntegerBits_b&gt; b) {
<span class="lineNum">     553 </span>            :   FixedPoint&lt;tRawType, tIntegerBits_a + tIntegerBits_b&gt; c;
<span class="lineNum">     554 </span><span class="lineCov">   26172112 :   c.raw() = SaturatingRoundingDoublingHighMul(a.raw(), b.raw());</span>
<span class="lineNum">     555 </span><span class="lineCov">   26172112 :   return c;</span>
<span class="lineNum">     556 </span>            : }
<span class="lineNum">     557 </span>            : 
<a name="558"><span class="lineNum">     558 </span>            : // Tweaking IntegerBits gives exact multiplication by a power of two.</a>
<span class="lineNum">     559 </span>            : template &lt;int tExponent, typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     560 </span><span class="lineCov">      88992 : FixedPoint&lt;tRawType, tExponent + tIntegerBits&gt; ExactMulByPot(</span>
<span class="lineNum">     561 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; a) {
<span class="lineNum">     562 </span>            :   FixedPoint&lt;tRawType, tExponent + tIntegerBits&gt; c;
<span class="lineNum">     563 </span><span class="lineCov">      88992 :   c.raw() = a.raw();</span>
<span class="lineNum">     564 </span><span class="lineCov">      88992 :   return c;</span>
<span class="lineNum">     565 </span>            : }
<span class="lineNum">     566 </span>            : 
<span class="lineNum">     567 </span>            : // If we want to leave IntegerBits fixed, then multiplication
<a name="568"><span class="lineNum">     568 </span>            : // by a power of two has to be saturating/rounding, not exact anymore.</a>
<span class="lineNum">     569 </span>            : template &lt;int tExponent, typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     570 </span><span class="lineCov">     217536 : FixedPoint&lt;tRawType, tIntegerBits&gt; SaturatingRoundingMultiplyByPOT(</span>
<span class="lineNum">     571 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; a) {
<span class="lineNum">     572 </span><span class="lineCov">     217536 :   return FixedPoint&lt;tRawType, tIntegerBits&gt;::FromRaw(</span>
<span class="lineNum">     573 </span><span class="lineCov">     435072 :       SaturatingRoundingMultiplyByPOT&lt;tExponent&gt;(a.raw()));</span>
<span class="lineNum">     574 </span>            : }
<span class="lineNum">     575 </span>            : 
<span class="lineNum">     576 </span>            : // Generic arithmetic operators.
<span class="lineNum">     577 </span>            : 
<span class="lineNum">     578 </span>            : #define MAKE_FIXEDPOINT_UNARY_FUNC(FuncName, ImplFuncName)                     \
<span class="lineNum">     579 </span>            :   template &lt;typename tRawType, int tIntegerBits&gt;                               \
<span class="lineNum">     580 </span>            :   FixedPoint&lt;tRawType, tIntegerBits&gt; FuncName(                                 \
<span class="lineNum">     581 </span>            :       FixedPoint&lt;tRawType, tIntegerBits&gt; a) {                                  \
<span class="lineNum">     582 </span>            :     return FixedPoint&lt;tRawType, tIntegerBits&gt;::FromRaw(ImplFuncName(a.raw())); \
<span class="lineNum">     583 </span>            :   }
<span class="lineNum">     584 </span>            : 
<span class="lineNum">     585 </span>            : #define MAKE_FIXEDPOINT_BINARY_FUNC(FuncName, ImplFuncName) \
<span class="lineNum">     586 </span>            :   template &lt;typename tRawType, int tIntegerBits&gt;            \
<span class="lineNum">     587 </span>            :   FixedPoint&lt;tRawType, tIntegerBits&gt; FuncName(              \
<span class="lineNum">     588 </span>            :       FixedPoint&lt;tRawType, tIntegerBits&gt; a,                 \
<span class="lineNum">     589 </span>            :       FixedPoint&lt;tRawType, tIntegerBits&gt; b) {               \
<span class="lineNum">     590 </span>            :     return FixedPoint&lt;tRawType, tIntegerBits&gt;::FromRaw(     \
<span class="lineNum">     591 </span>            :         ImplFuncName(a.raw(), b.raw()));                    \
<a name="592"><span class="lineNum">     592 </span>            :   }</a>
<span class="lineNum">     593 </span>            : 
<a name="594"><span class="lineNum">     594 </span><span class="lineCov">     138432 : MAKE_FIXEDPOINT_UNARY_FUNC(operator-, Neg)</span></a>
<a name="595"><span class="lineNum">     595 </span>            : MAKE_FIXEDPOINT_UNARY_FUNC(operator~, BitNot)</a>
<a name="596"><span class="lineNum">     596 </span><span class="lineCov">     751488 : MAKE_FIXEDPOINT_BINARY_FUNC(operator+, Add)</span></a>
<span class="lineNum">     597 </span><span class="lineCov">     622944 : MAKE_FIXEDPOINT_BINARY_FUNC(operator-, Sub)</span>
<span class="lineNum">     598 </span><span class="lineCov">     103824 : MAKE_FIXEDPOINT_BINARY_FUNC(operator&amp;, BitAnd)</span>
<a name="599"><span class="lineNum">     599 </span>            : MAKE_FIXEDPOINT_BINARY_FUNC(operator^, BitXor)</a>
<span class="lineNum">     600 </span>            : MAKE_FIXEDPOINT_BINARY_FUNC(operator|, BitOr)
<span class="lineNum">     601 </span><span class="lineCov">      79104 : MAKE_FIXEDPOINT_BINARY_FUNC(RoundingHalfSum, RoundingHalfSum)</span>
<span class="lineNum">     602 </span>            : 
<span class="lineNum">     603 </span>            : #undef MAKE_FIXEDPOINT_UNARY_FUNC
<span class="lineNum">     604 </span>            : #undef MAKE_FIXEDPOINT_BINARY_FUNC
<span class="lineNum">     605 </span>            : 
<span class="lineNum">     606 </span>            : #define MAKE_FIXEDPOINT_UNARY_FUNC_RETURNING_RAW(FuncName)  \
<span class="lineNum">     607 </span>            :   template &lt;typename tRawType, int tIntegerBits&gt;            \
<span class="lineNum">     608 </span>            :   tRawType FuncName(FixedPoint&lt;tRawType, tIntegerBits&gt; a) { \
<span class="lineNum">     609 </span>            :     return FuncName(a.raw());                               \
<span class="lineNum">     610 </span>            :   }
<span class="lineNum">     611 </span>            : 
<span class="lineNum">     612 </span>            : #define MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(FuncName) \
<span class="lineNum">     613 </span>            :   template &lt;typename tRawType, int tIntegerBits&gt;            \
<span class="lineNum">     614 </span>            :   tRawType FuncName(FixedPoint&lt;tRawType, tIntegerBits&gt; a,   \
<span class="lineNum">     615 </span>            :                     FixedPoint&lt;tRawType, tIntegerBits&gt; b) { \
<span class="lineNum">     616 </span>            :     return FuncName(a.raw(), b.raw());                      \
<a name="617"><span class="lineNum">     617 </span>            :   }</a>
<span class="lineNum">     618 </span>            : 
<span class="lineNum">     619 </span><span class="lineCov">     173040 : MAKE_FIXEDPOINT_UNARY_FUNC_RETURNING_RAW(MaskIfZero)</span>
<span class="lineNum">     620 </span>            : MAKE_FIXEDPOINT_UNARY_FUNC_RETURNING_RAW(MaskIfNonZero)
<a name="621"><span class="lineNum">     621 </span>            : MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfEqual)</a>
<span class="lineNum">     622 </span>            : MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfNotEqual)
<a name="623"><span class="lineNum">     623 </span><span class="lineCov">      34608 : MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfGreaterThan)</span></a>
<span class="lineNum">     624 </span>            : MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfGreaterThanOrEqual)
<span class="lineNum">     625 </span><span class="lineCov">      54384 : MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfLessThan)</span>
<span class="lineNum">     626 </span>            : MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfLessThanOrEqual)
<span class="lineNum">     627 </span>            : 
<span class="lineNum">     628 </span>            : #undef MAKE_FIXEDPOINT_UNARY_FUNC_RETURNING_RAW
<span class="lineNum">     629 </span>            : #undef MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW
<a name="630"><span class="lineNum">     630 </span>            : </a>
<span class="lineNum">     631 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     632 </span><span class="lineCov">     860256 : FixedPoint&lt;tRawType, tIntegerBits&gt; SelectUsingMask(</span>
<span class="lineNum">     633 </span>            :     tRawType if_mask, FixedPoint&lt;tRawType, tIntegerBits&gt; then_val,
<span class="lineNum">     634 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; else_val) {
<span class="lineNum">     635 </span><span class="lineCov">    1318224 :   return FixedPoint&lt;tRawType, tIntegerBits&gt;::FromRaw(</span>
<span class="lineNum">     636 </span><span class="lineCov">    2178480 :       SelectUsingMask(if_mask, then_val.raw(), else_val.raw()));</span>
<span class="lineNum">     637 </span>            : }
<a name="638"><span class="lineNum">     638 </span>            : </a>
<span class="lineNum">     639 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     640 </span><span class="lineCov">     978912 : bool operator==(FixedPoint&lt;tRawType, tIntegerBits&gt; a,</span>
<span class="lineNum">     641 </span>            :                 FixedPoint&lt;tRawType, tIntegerBits&gt; b) {
<span class="lineNum">     642 </span><span class="lineCov">     978912 :   return All(MaskIfEqual(a.raw(), b.raw()));</span>
<span class="lineNum">     643 </span>            : }
<span class="lineNum">     644 </span>            : 
<span class="lineNum">     645 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     646 </span>            : bool operator!=(FixedPoint&lt;tRawType, tIntegerBits&gt; a,
<span class="lineNum">     647 </span>            :                 FixedPoint&lt;tRawType, tIntegerBits&gt; b) {
<span class="lineNum">     648 </span>            :   return !(a == b);
<span class="lineNum">     649 </span>            : }
<span class="lineNum">     650 </span>            : 
<span class="lineNum">     651 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     652 </span>            : FixedPoint&lt;tRawType, tIntegerBits&gt; SaturatingAdd(
<span class="lineNum">     653 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; a,
<span class="lineNum">     654 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; b) {
<span class="lineNum">     655 </span>            :   return FixedPoint&lt;tRawType, tIntegerBits&gt;::FromRaw(
<span class="lineNum">     656 </span>            :       SaturatingAdd(a.raw(), b.raw()));
<span class="lineNum">     657 </span>            : }
<a name="658"><span class="lineNum">     658 </span>            : </a>
<span class="lineNum">     659 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     660 </span><span class="lineCov">     108768 : FixedPoint&lt;tRawType, tIntegerBits&gt; AddSaturatingIf16Bit(</span>
<span class="lineNum">     661 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; a,
<span class="lineNum">     662 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; b) {
<span class="lineNum">     663 </span><span class="lineCov">     166672 :   return FixedPoint&lt;tRawType, tIntegerBits&gt;::FromRaw(</span>
<span class="lineNum">     664 </span><span class="lineCov">     275440 :       AddSaturatingIf16Bit(a.raw(), b.raw()));</span>
<span class="lineNum">     665 </span>            : }
<span class="lineNum">     666 </span>            : 
<a name="667"><span class="lineNum">     667 </span>            : // Conversion to floating-point.</a>
<span class="lineNum">     668 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     669 </span><span class="lineCov">   49199840 : double ToDouble(FixedPoint&lt;tRawType, tIntegerBits&gt; x) {</span>
<span class="lineNum">     670 </span>            :   static_assert(FixedPointRawTypeTraits&lt;tRawType&gt;::kLanes == 1,
<span class="lineNum">     671 </span>            :                 &quot;not applicable to SIMD types&quot;);
<span class="lineNum">     672 </span>            :   typedef FixedPoint&lt;tRawType, tIntegerBits&gt; F;
<span class="lineNum">     673 </span><span class="lineCov">   49199840 :   return x.raw() / static_cast&lt;double&gt;(1ll &lt;&lt; F::kFractionalBits);</span>
<span class="lineNum">     674 </span>            : }
<span class="lineNum">     675 </span>            : 
<span class="lineNum">     676 </span>            : // Rescale changes the number of IntegerBits and updates the underlying
<a name="677"><span class="lineNum">     677 </span>            : // raw integer value accordingly.</a>
<span class="lineNum">     678 </span>            : template &lt;int tIntegerBitsDst, typename tRawType, int tIntegerBitsSrc&gt;
<span class="lineNum">     679 </span><span class="lineCov">     437544 : FixedPoint&lt;tRawType, tIntegerBitsDst&gt; Rescale(</span>
<span class="lineNum">     680 </span>            :     FixedPoint&lt;tRawType, tIntegerBitsSrc&gt; x) {
<span class="lineNum">     681 </span>            :   static constexpr int kExponent = tIntegerBitsSrc - tIntegerBitsDst;
<span class="lineNum">     682 </span>            :   FixedPoint&lt;tRawType, tIntegerBitsDst&gt; result;
<span class="lineNum">     683 </span><span class="lineCov">     437544 :   result.raw() = SaturatingRoundingMultiplyByPOT&lt;kExponent&gt;(x.raw());</span>
<span class="lineNum">     684 </span><span class="lineCov">     437544 :   return result;</span>
<span class="lineNum">     685 </span>            : }
<span class="lineNum">     686 </span>            : 
<span class="lineNum">     687 </span>            : // CheckedFixedPointConstant allows to specify fixed-point constants
<span class="lineNum">     688 </span>            : // initialized as real numbers, in a way that does not compile floating-point
<span class="lineNum">     689 </span>            : // arithmetic in production code, yet still checks agreement with the
<span class="lineNum">     690 </span>            : // floating-point expressions when asserts are enabled.
<span class="lineNum">     691 </span>            : //
<span class="lineNum">     692 </span>            : // The raw integer value provided is always a int32, encoding a 32-bit
<span class="lineNum">     693 </span>            : // fixed-point value, regardless of the actual Scalar type. This allows
<span class="lineNum">     694 </span>            : // writing generic code that applies just as well to the 32-bit and 16-bit
<span class="lineNum">     695 </span>            : // cases. In the 16-bit case, the raw integer value is internally
<a name="696"><span class="lineNum">     696 </span>            : // rounding-shifted by 16 bits to the right.</a>
<span class="lineNum">     697 </span>            : template &lt;typename FixedPointType&gt;
<span class="lineNum">     698 </span><span class="lineCov">     959136 : inline typename FixedPointType::ScalarRawType RescaleConstantInitializer(</span>
<span class="lineNum">     699 </span>            :     std::int32_t int32_value) {
<span class="lineNum">     700 </span>            :   typedef typename FixedPointType::ScalarRawType ScalarRawType;
<span class="lineNum">     701 </span>            :   static constexpr int ScalarTypeBits = 8 * sizeof(ScalarRawType);
<span class="lineNum">     702 </span>            :   return static_cast&lt;ScalarRawType&gt;(
<span class="lineNum">     703 </span><span class="lineCov">     959136 :       RoundingDivideByPOT&lt;std::int32_t&gt;(int32_value, 32 - ScalarTypeBits));</span>
<span class="lineNum">     704 </span>            : }
<a name="705"><span class="lineNum">     705 </span>            : #ifdef GEMMLOWP_ENABLE_FIXEDPOINT_CONSTANTS_CHECKS</a>
<span class="lineNum">     706 </span>            : template &lt;typename FixedPointType&gt;
<span class="lineNum">     707 </span><span class="lineCov">     959136 : FixedPointType CheckedFixedPointConstant(std::int32_t raw_value,</span>
<span class="lineNum">     708 </span>            :                                          double double_value) {
<span class="lineNum">     709 </span><span class="lineCov">     959136 :   const FixedPointType result = FixedPointType::FromScalarRaw(raw_value);</span>
<span class="lineNum">     710 </span><span class="lineCov">     959136 :   assert(result == FixedPointType::FromDouble(double_value));</span>
<span class="lineNum">     711 </span><span class="lineCov">     959136 :   return result;</span>
<span class="lineNum">     712 </span>            : }
<span class="lineNum">     713 </span>            : #define GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(FixedPointType,                   \
<span class="lineNum">     714 </span>            :                                              ScalarRawInt32Value, DoubleValue) \
<span class="lineNum">     715 </span>            :   (gemmlowp::CheckedFixedPointConstant&lt;FixedPointType&gt;(                        \
<span class="lineNum">     716 </span>            :       gemmlowp::RescaleConstantInitializer&lt;FixedPointType&gt;(                    \
<span class="lineNum">     717 </span>            :           ScalarRawInt32Value),                                                \
<span class="lineNum">     718 </span>            :       DoubleValue))
<span class="lineNum">     719 </span>            : 
<span class="lineNum">     720 </span>            : #else
<span class="lineNum">     721 </span>            : #define GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(FixedPointType,                   \
<span class="lineNum">     722 </span>            :                                              ScalarRawInt32Value, DoubleValue) \
<span class="lineNum">     723 </span>            :   (FixedPointType::FromScalarRaw(                                              \
<span class="lineNum">     724 </span>            :       gemmlowp::RescaleConstantInitializer&lt;FixedPointType&gt;(                    \
<span class="lineNum">     725 </span>            :           ScalarRawInt32Value)))
<span class="lineNum">     726 </span>            : #endif
<span class="lineNum">     727 </span>            : 
<span class="lineNum">     728 </span>            : // Implementation of exponential function.
<span class="lineNum">     729 </span>            : 
<a name="730"><span class="lineNum">     730 </span>            : // Returns exp(x) for x in [-1/4, 0).</a>
<span class="lineNum">     731 </span>            : template &lt;typename tRawType&gt;
<span class="lineNum">     732 </span><span class="lineCov">     108768 : FixedPoint&lt;tRawType, 0&gt; exp_on_interval_between_negative_one_quarter_and_0_excl(</span>
<span class="lineNum">     733 </span>            :     FixedPoint&lt;tRawType, 0&gt; a) {
<span class="lineNum">     734 </span>            :   typedef FixedPoint&lt;tRawType, 0&gt; F;
<span class="lineNum">     735 </span>            :   const F constant_term =
<span class="lineNum">     736 </span><span class="lineCov">     108768 :       GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F, 1895147668, std::exp(-1.0 / 8.0));</span>
<span class="lineNum">     737 </span>            :   const F constant_1_over_3 =
<span class="lineNum">     738 </span><span class="lineCov">     108768 :       GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F, 715827883, 1.0 / 3.0);</span>
<span class="lineNum">     739 </span>            :   // We're evaluating a Taylor expansion around -1/8, so we do the change of
<span class="lineNum">     740 </span>            :   // variable: x = a + 1/8.
<span class="lineNum">     741 </span>            :   // In fixed-point with 0 integer bits, 1/8 is represented by 1 &lt;&lt; 28.
<span class="lineNum">     742 </span><span class="lineCov">     108768 :   F x = a + F::template ConstantPOT&lt;-3&gt;();</span>
<span class="lineNum">     743 </span><span class="lineCov">     108768 :   F x2 = x * x;</span>
<span class="lineNum">     744 </span><span class="lineCov">     108768 :   F x3 = x2 * x;</span>
<span class="lineNum">     745 </span><span class="lineCov">     108768 :   F x4 = x2 * x2;</span>
<span class="lineNum">     746 </span><span class="lineCov">     108768 :   F x4_over_4 = SaturatingRoundingMultiplyByPOT&lt;-2&gt;(x4);</span>
<span class="lineNum">     747 </span>            :   F x4_over_24_plus_x3_over_6_plus_x2_over_2 =
<span class="lineNum">     748 </span><span class="lineCov">     108768 :       SaturatingRoundingMultiplyByPOT&lt;-1&gt;(</span>
<span class="lineNum">     749 </span><span class="lineCov">     108768 :           ((x4_over_4 + x3) * constant_1_over_3) + x2);</span>
<span class="lineNum">     750 </span><span class="lineCov">     108768 :   return AddSaturatingIf16Bit(</span>
<span class="lineNum">     751 </span>            :       constant_term,
<span class="lineNum">     752 </span><span class="lineCov">     108768 :       constant_term * (x + x4_over_24_plus_x3_over_6_plus_x2_over_2));</span>
<span class="lineNum">     753 </span>            : }
<span class="lineNum">     754 </span>            : 
<a name="755"><span class="lineNum">     755 </span>            : // Returns exp(x) for x &lt; 0.</a>
<span class="lineNum">     756 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     757 </span><span class="lineCov">     103824 : FixedPoint&lt;tRawType, 0&gt; exp_on_negative_values(</span>
<span class="lineNum">     758 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; a) {
<span class="lineNum">     759 </span>            :   typedef FixedPoint&lt;tRawType, tIntegerBits&gt; InputF;
<span class="lineNum">     760 </span>            :   typedef FixedPoint&lt;tRawType, 0&gt; ResultF;
<span class="lineNum">     761 </span>            :   static constexpr int kFractionalBits = InputF::kFractionalBits;
<span class="lineNum">     762 </span>            :   static constexpr int kIntegerBits = InputF::kIntegerBits;
<span class="lineNum">     763 </span><span class="lineCov">     103824 :   const InputF kOneQuarter = InputF::template ConstantPOT&lt;-2&gt;();</span>
<span class="lineNum">     764 </span><span class="lineCov">     103824 :   InputF mask = kOneQuarter - InputF::FromScalarRaw(1);</span>
<span class="lineNum">     765 </span><span class="lineCov">     103824 :   InputF a_mod_quarter_minus_one_quarter = (a &amp; mask) - kOneQuarter;</span>
<span class="lineNum">     766 </span><span class="lineCov">     103824 :   ResultF result = exp_on_interval_between_negative_one_quarter_and_0_excl(</span>
<span class="lineNum">     767 </span><span class="lineCov">     103824 :       Rescale&lt;0&gt;(a_mod_quarter_minus_one_quarter));</span>
<span class="lineNum">     768 </span><span class="lineCov">     103824 :   tRawType remainder = (a_mod_quarter_minus_one_quarter - a).raw();</span>
<span class="lineNum">     769 </span>            : 
<span class="lineNum">     770 </span>            : #define GEMMLOWP_EXP_BARREL_SHIFTER(Exponent, FixedPointMultiplier)         \
<span class="lineNum">     771 </span>            :   if (kIntegerBits &gt; Exponent) {                                            \
<span class="lineNum">     772 </span>            :     const ResultF kMultiplier = GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(       \
<span class="lineNum">     773 </span>            :         ResultF, FixedPointMultiplier, std::exp(-std::pow(2.0, Exponent))); \
<span class="lineNum">     774 </span>            :     static constexpr int kShiftAmount =                                     \
<span class="lineNum">     775 </span>            :         kIntegerBits &gt; Exponent ? kFractionalBits + Exponent : 0;           \
<span class="lineNum">     776 </span>            :     result = SelectUsingMask(                                               \
<span class="lineNum">     777 </span>            :         MaskIfNonZero(BitAnd(remainder, Dup&lt;tRawType&gt;(1 &lt;&lt; kShiftAmount))), \
<span class="lineNum">     778 </span>            :         result * kMultiplier, result);                                      \
<span class="lineNum">     779 </span>            :   }
<span class="lineNum">     780 </span>            : 
<span class="lineNum">     781 </span>            :   // Constants below are Q0 representations of negative exp fractionals:
<span class="lineNum">     782 </span><span class="lineCov">     103824 :   GEMMLOWP_EXP_BARREL_SHIFTER(-2, 1672461947);  // exp(-1/4)</span>
<span class="lineNum">     783 </span><span class="lineCov">     103824 :   GEMMLOWP_EXP_BARREL_SHIFTER(-1, 1302514674);  // exp(-1/2)</span>
<span class="lineNum">     784 </span><span class="lineCov">      93936 :   GEMMLOWP_EXP_BARREL_SHIFTER(+0, 790015084);   // exp(-1)</span>
<span class="lineNum">     785 </span><span class="lineCov">      79104 :   GEMMLOWP_EXP_BARREL_SHIFTER(+1, 290630308);   // exp(-2)</span>
<span class="lineNum">     786 </span><span class="lineCov">      64272 :   GEMMLOWP_EXP_BARREL_SHIFTER(+2, 39332535);    // exp(-4)</span>
<span class="lineNum">     787 </span><span class="lineCov">      49440 :   GEMMLOWP_EXP_BARREL_SHIFTER(+3, 720401);      // exp(-8)</span>
<span class="lineNum">     788 </span><span class="lineCov">      34608 :   GEMMLOWP_EXP_BARREL_SHIFTER(+4, 242);         // exp(-16)</span>
<span class="lineNum">     789 </span>            : 
<span class="lineNum">     790 </span>            : #undef GEMMLOWP_EXP_BARREL_SHIFTER
<span class="lineNum">     791 </span>            : 
<span class="lineNum">     792 </span>            :   static constexpr int clampB = kIntegerBits &gt; 5 ? 36 - kIntegerBits : 0;
<span class="lineNum">     793 </span>            :   if (kIntegerBits &gt; 5) {
<span class="lineNum">     794 </span>            :     const InputF clamp =
<span class="lineNum">     795 </span><span class="lineCov">      19776 :         GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(InputF, -(1 &lt;&lt; clampB), -32.0);</span>
<span class="lineNum">     796 </span><span class="lineCov">      19776 :     result = SelectUsingMask(MaskIfLessThan(a, clamp), ResultF::Zero(), result);</span>
<span class="lineNum">     797 </span>            :   }
<span class="lineNum">     798 </span>            : 
<span class="lineNum">     799 </span><span class="lineCov">     103824 :   result = SelectUsingMask(MaskIfZero(a), ResultF::One(), result);</span>
<span class="lineNum">     800 </span><span class="lineCov">     103824 :   return result;</span>
<span class="lineNum">     801 </span>            : }
<span class="lineNum">     802 </span>            : 
<span class="lineNum">     803 </span>            : // Implementation of tanh: (1 - exp(-2x)) / (1 + exp(-2x)).
<span class="lineNum">     804 </span>            : 
<a name="805"><span class="lineNum">     805 </span>            : // Returns (1 - x) / (1 + x) for x in (0, 1).</a>
<span class="lineNum">     806 </span>            : template &lt;typename tRawType&gt;
<span class="lineNum">     807 </span><span class="lineCov">      39552 : FixedPoint&lt;tRawType, 0&gt; one_minus_x_over_one_plus_x_for_x_in_0_1(</span>
<span class="lineNum">     808 </span>            :     FixedPoint&lt;tRawType, 0&gt; a) {
<span class="lineNum">     809 </span>            :   typedef FixedPoint&lt;tRawType, 0&gt; F0;
<span class="lineNum">     810 </span>            :   typedef FixedPoint&lt;tRawType, 2&gt; F2;
<span class="lineNum">     811 </span><span class="lineCov">      39552 :   F0 half_denominator = RoundingHalfSum(a, F0::One());</span>
<span class="lineNum">     812 </span>            :   // Newton-Raphson division
<span class="lineNum">     813 </span>            :   // https://en.wikipedia.org/wiki/Division_algorithm#Newton.E2.80.93Raphson_division
<span class="lineNum">     814 </span>            :   // Refer to that page for the logic behind the 48/17 and 32/17 constants.
<span class="lineNum">     815 </span>            :   const F2 constant_48_over_17 =
<span class="lineNum">     816 </span><span class="lineCov">      39552 :       GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F2, 1515870810, 48.0 / 17.0);</span>
<span class="lineNum">     817 </span>            :   const F2 constant_neg_32_over_17 =
<span class="lineNum">     818 </span><span class="lineCov">      39552 :       GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F2, -1010580540, -32.0 / 17.0);</span>
<span class="lineNum">     819 </span><span class="lineCov">      39552 :   F2 x = constant_48_over_17 + half_denominator * constant_neg_32_over_17;</span>
<span class="lineNum">     820 </span><span class="lineCov">     158208 :   for (int i = 0; i &lt; 3; i++) {</span>
<span class="lineNum">     821 </span><span class="lineCov">     118656 :     F2 half_denominator_times_x = half_denominator * x;</span>
<span class="lineNum">     822 </span>            :     F2 one_minus_half_denominator_times_x =
<span class="lineNum">     823 </span><span class="lineCov">     118656 :         F2::One() - half_denominator_times_x;</span>
<span class="lineNum">     824 </span><span class="lineCov">     118656 :     x = x + Rescale&lt;2&gt;(x * one_minus_half_denominator_times_x);</span>
<span class="lineNum">     825 </span>            :   }
<span class="lineNum">     826 </span><span class="lineCov">      39552 :   return Rescale&lt;0&gt;(x - F2::One());</span>
<span class="lineNum">     827 </span>            : }
<span class="lineNum">     828 </span>            : 
<a name="829"><span class="lineNum">     829 </span>            : // Returns -tanh(x) for x &lt; 0.</a>
<span class="lineNum">     830 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     831 </span><span class="lineCov">      34608 : FixedPoint&lt;tRawType, 0&gt; neg_tanh_on_negative_values(</span>
<span class="lineNum">     832 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; a) {
<span class="lineNum">     833 </span><span class="lineCov">      34608 :   return one_minus_x_over_one_plus_x_for_x_in_0_1(</span>
<span class="lineNum">     834 </span><span class="lineCov">      34608 :       exp_on_negative_values(ExactMulByPot&lt;1&gt;(a)));</span>
<span class="lineNum">     835 </span>            : }
<span class="lineNum">     836 </span>            : 
<a name="837"><span class="lineNum">     837 </span>            : // Returns tanh(x) for any x.</a>
<span class="lineNum">     838 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     839 </span><span class="lineCov">      34608 : FixedPoint&lt;tRawType, 0&gt; tanh(FixedPoint&lt;tRawType, tIntegerBits&gt; a) {</span>
<span class="lineNum">     840 </span>            :   typedef FixedPoint&lt;tRawType, tIntegerBits&gt; InputF;
<span class="lineNum">     841 </span>            :   typedef FixedPoint&lt;tRawType, 0&gt; ResultF;
<span class="lineNum">     842 </span><span class="lineCov">      34608 :   tRawType mask_if_negative = MaskIfLessThan(a, InputF::Zero());</span>
<span class="lineNum">     843 </span><span class="lineCov">      34608 :   tRawType mask_if_zero = MaskIfZero(a);</span>
<span class="lineNum">     844 </span><span class="lineCov">      34608 :   InputF n = SelectUsingMask(mask_if_negative, a, -a);</span>
<span class="lineNum">     845 </span><span class="lineCov">      34608 :   ResultF t = neg_tanh_on_negative_values(n);</span>
<span class="lineNum">     846 </span><span class="lineCov">      34608 :   return SelectUsingMask(mask_if_zero, ResultF::Zero(),</span>
<span class="lineNum">     847 </span><span class="lineCov">      34608 :                          SelectUsingMask(mask_if_negative, -t, t));</span>
<span class="lineNum">     848 </span>            : }
<span class="lineNum">     849 </span>            : 
<span class="lineNum">     850 </span>            : // Implementation of logistic function.
<span class="lineNum">     851 </span>            : 
<a name="852"><span class="lineNum">     852 </span>            : // Returns 1 / (1 + x) for x in (0, 1).</a>
<span class="lineNum">     853 </span>            : template &lt;typename tRawType&gt;
<span class="lineNum">     854 </span><span class="lineCov">      39552 : FixedPoint&lt;tRawType, 0&gt; one_over_one_plus_x_for_x_in_0_1(</span>
<span class="lineNum">     855 </span>            :     FixedPoint&lt;tRawType, 0&gt; a) {
<span class="lineNum">     856 </span>            :   typedef FixedPoint&lt;tRawType, 0&gt; F0;
<span class="lineNum">     857 </span>            :   typedef FixedPoint&lt;tRawType, 2&gt; F2;
<span class="lineNum">     858 </span><span class="lineCov">      39552 :   F0 half_denominator = RoundingHalfSum(a, F0::One());</span>
<span class="lineNum">     859 </span>            :   // Newton-Raphson division
<span class="lineNum">     860 </span>            :   // https://en.wikipedia.org/wiki/Division_algorithm#Newton.E2.80.93Raphson_division
<span class="lineNum">     861 </span>            :   // Refer to that page for the logic behind the 48/17 and 32/17 constants.
<span class="lineNum">     862 </span>            :   const F2 constant_48_over_17 =
<span class="lineNum">     863 </span><span class="lineCov">      39552 :       GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F2, 1515870810, 48.0 / 17.0);</span>
<span class="lineNum">     864 </span>            :   const F2 constant_neg_32_over_17 =
<span class="lineNum">     865 </span><span class="lineCov">      39552 :       GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F2, -1010580540, -32.0 / 17.0);</span>
<span class="lineNum">     866 </span><span class="lineCov">      39552 :   F2 x = constant_48_over_17 + half_denominator * constant_neg_32_over_17;</span>
<span class="lineNum">     867 </span><span class="lineCov">     158208 :   for (int i = 0; i &lt; 3; i++) {</span>
<span class="lineNum">     868 </span><span class="lineCov">     118656 :     F2 half_denominator_times_x = half_denominator * x;</span>
<span class="lineNum">     869 </span>            :     F2 one_minus_half_denominator_times_x =
<span class="lineNum">     870 </span><span class="lineCov">     118656 :         F2::One() - half_denominator_times_x;</span>
<span class="lineNum">     871 </span><span class="lineCov">     118656 :     x = x + Rescale&lt;2&gt;(x * one_minus_half_denominator_times_x);</span>
<span class="lineNum">     872 </span>            :   }
<span class="lineNum">     873 </span><span class="lineCov">      39552 :   return Rescale&lt;0&gt;(ExactMulByPot&lt;-1&gt;(x));</span>
<span class="lineNum">     874 </span>            : }
<span class="lineNum">     875 </span>            : 
<a name="876"><span class="lineNum">     876 </span>            : // Returns logistic(x) = 1 / (1 + exp(-x)) for x &gt; 0.</a>
<span class="lineNum">     877 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     878 </span><span class="lineCov">      34608 : FixedPoint&lt;tRawType, 0&gt; logistic_on_positive_values(</span>
<span class="lineNum">     879 </span>            :     FixedPoint&lt;tRawType, tIntegerBits&gt; a) {
<span class="lineNum">     880 </span><span class="lineCov">      34608 :   return one_over_one_plus_x_for_x_in_0_1(exp_on_negative_values(-a));</span>
<span class="lineNum">     881 </span>            : }
<span class="lineNum">     882 </span>            : 
<a name="883"><span class="lineNum">     883 </span>            : // Returns logistic(x) = 1 / (1 + exp(-x)) for any x.</a>
<span class="lineNum">     884 </span>            : template &lt;typename tRawType, int tIntegerBits&gt;
<span class="lineNum">     885 </span><span class="lineCov">      34608 : FixedPoint&lt;tRawType, 0&gt; logistic(FixedPoint&lt;tRawType, tIntegerBits&gt; a) {</span>
<span class="lineNum">     886 </span>            :   typedef FixedPoint&lt;tRawType, tIntegerBits&gt; InputF;
<span class="lineNum">     887 </span>            :   typedef FixedPoint&lt;tRawType, 0&gt; ResultF;
<span class="lineNum">     888 </span><span class="lineCov">      34608 :   tRawType mask_if_positive = MaskIfGreaterThan(a, InputF::Zero());</span>
<span class="lineNum">     889 </span><span class="lineCov">      34608 :   tRawType mask_if_zero = MaskIfZero(a);</span>
<span class="lineNum">     890 </span><span class="lineCov">      34608 :   InputF abs_input = SelectUsingMask(mask_if_positive, a, -a);</span>
<span class="lineNum">     891 </span><span class="lineCov">      34608 :   ResultF result_if_positive = logistic_on_positive_values(abs_input);</span>
<span class="lineNum">     892 </span><span class="lineCov">      34608 :   ResultF result_if_negative = ResultF::One() - result_if_positive;</span>
<span class="lineNum">     893 </span>            :   const ResultF one_half =
<span class="lineNum">     894 </span><span class="lineCov">      34608 :       GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(ResultF, 1 &lt;&lt; 30, 0.5);</span>
<span class="lineNum">     895 </span><span class="lineCov">      34608 :   return SelectUsingMask(mask_if_zero, one_half,</span>
<span class="lineNum">     896 </span>            :                          SelectUsingMask(mask_if_positive, result_if_positive,
<span class="lineNum">     897 </span><span class="lineCov">      34608 :                                          result_if_negative));</span>
<span class="lineNum">     898 </span>            : }
<span class="lineNum">     899 </span>            : 
<span class="lineNum">     900 </span>            : }  // end namespace gemmlowp
<span class="lineNum">     901 </span>            : 
<span class="lineNum">     902 </span>            : #ifdef GEMMLOWP_NEON
<span class="lineNum">     903 </span>            : #include &quot;./fixedpoint_neon.h&quot;
<span class="lineNum">     904 </span>            : #elif defined(GEMMLOWP_AVX2)
<span class="lineNum">     905 </span>            : #include &quot;./fixedpoint_avx.h&quot;
<span class="lineNum">     906 </span>            : #elif defined(GEMMLOWP_SSE4)
<span class="lineNum">     907 </span>            : #include &quot;./fixedpoint_sse.h&quot;
<span class="lineNum">     908 </span>            : #elif defined(GEMMLOWP_MSA)
<span class="lineNum">     909 </span>            : #include &quot;./fixedpoint_msa.h&quot;
<span class="lineNum">     910 </span>            : #endif
<span class="lineNum">     911 </span>            : 
<span class="lineNum">     912 </span>            : #endif  // GEMMLOWP_INTERNAL_FIXEDPOINT_H_
</pre>
      </td>
    </tr>
  </table>
  <br>

  <table width="100%" border=0 cellspacing=0 cellpadding=0>
    <tr><td class="ruler"><img src="../../../../glass.png" width=3 height=3 alt=""></td></tr>
    <tr><td class="versionInfo">Generated by: <a href="http://ltp.sourceforge.net/coverage/lcov.php" target="_parent">LCOV version 1.12</a></td></tr>
  </table>
  <br>

</body>
</html>
